<?xml version="1.0" encoding="ascii"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
          "DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
  <title>pybindgen.typehandlers.ctypeparser.tokenizer</title>
  <link rel="stylesheet" href="epydoc.css" type="text/css" />
  <script type="text/javascript" src="epydoc.js"></script>
</head>

<body bgcolor="white" text="black" link="blue" vlink="#204080"
      alink="#204080">
<!-- ==================== NAVIGATION BAR ==================== -->
<table class="navbar" border="0" width="100%" cellpadding="0"
       bgcolor="#a0c0ff" cellspacing="0">
  <tr valign="middle">
  <!-- Home link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="pybindgen-module.html">Home</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Tree link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="module-tree.html">Trees</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Index link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="identifier-index.html">Indices</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Help link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="help.html">Help</a>&nbsp;&nbsp;&nbsp;</th>

      <th class="navbar" width="100%"></th>
  </tr>
</table>
<table width="100%" cellpadding="0" cellspacing="0">
  <tr valign="top">
    <td width="100%">
      <span class="breadcrumbs">
        <a href="pybindgen-module.html">Package&nbsp;pybindgen</a> ::
        <a href="pybindgen.typehandlers-module.html">Package&nbsp;typehandlers</a> ::
        <a href="pybindgen.typehandlers.ctypeparser-module.html">Package&nbsp;ctypeparser</a> ::
        Module&nbsp;tokenizer
      </span>
    </td>
    <td>
      <table cellpadding="0" cellspacing="0">
        <!-- hide/show private -->
        <tr><td align="right"><span class="options"
            >[<a href="frames.html" target="_top">frames</a
            >]&nbsp;|&nbsp;<a href="pybindgen.typehandlers.ctypeparser.tokenizer-module.html"
            target="_top">no&nbsp;frames</a>]</span></td></tr>
      </table>
    </td>
  </tr>
</table>
<!-- ==================== MODULE DESCRIPTION ==================== -->
<h1 class="epydoc">Module tokenizer</h1><p class="nomargin-top"><span class="codelink"><a href="pybindgen.typehandlers.ctypeparser.tokenizer-pysrc.html">source&nbsp;code</a></span></p>
<p>Tokenize C++ source code.</p>

<hr />
<div class="fields">      <p><strong>Author:</strong>
        nnorwitz@google.com (Neal Norwitz); minor changes by Gustavo J. A. 
        M. Carneiro (gjcarneiro@gmail.com)
      </p>
</div><!-- ==================== CLASSES ==================== -->
<a name="section-Classes"></a>
<table class="summary" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr bgcolor="#70b0f0" class="table-header">
  <td align="left" colspan="2" class="table-header">
    <span class="table-header">Classes</span></td>
</tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a href="pybindgen.typehandlers.ctypeparser.tokenizer.Token-class.html" class="summary-name">Token</a><br />
      Data container to represent a C++ token.
    </td>
  </tr>
</table>
<!-- ==================== FUNCTIONS ==================== -->
<a name="section-Functions"></a>
<table class="summary" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr bgcolor="#70b0f0" class="table-header">
  <td align="left" colspan="2" class="table-header">
    <span class="table-header">Functions</span></td>
</tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
      <table width="100%" cellpadding="0" cellspacing="0" border="0">
        <tr>
          <td><span class="summary-sig"><a href="pybindgen.typehandlers.ctypeparser.tokenizer-module.html#GetTokens" class="summary-sig-name">GetTokens</a>(<span class="summary-sig-arg">source</span>)</span><br />
      Returns a sequence of Tokens.</td>
          <td align="right" valign="top">
            <span class="codelink"><a href="pybindgen.typehandlers.ctypeparser.tokenizer-pysrc.html#GetTokens">source&nbsp;code</a></span>
            <br /><span class="codelink"><a href="javascript:void(0);" onclick="toggleCallGraph('call_graph_for_gettokens-summary-div');return false;">call&nbsp;graph</a></span>&nbsp;
          </td>
        </tr>
      </table>
      <div style="display:none" id="call_graph_for_gettokens-summary-div"><center>
<table border="0" cellpadding="0" cellspacing="0">
  <tr><td><center>  <map id="call_graph_for_gettokens" name="call_graph_for_gettokens">
<area shape="rect" href="pybindgen.typehandlers.ctypeparser-module.html#parse_type" title="parse_type()" alt="" coords="5,5,141,37"/>
<area shape="rect" href="pybindgen.typehandlers.ctypeparser.tokenizer-module.html#GetTokens" title="GetTokens()" alt="" coords="191,5,321,37"/>
<area shape="rect" href="pybindgen.typehandlers.ctypeparser.tokenizer.Token-class.html#__init__" title="Token.__init__()" alt="" coords="372,5,535,37"/>
</map>
  <img src="call_graph_for_gettokens.gif" alt='' usemap="#call_graph_for_gettokens" ismap="ismap" class="graph-without-title" />
</center></td></tr>
  <tr><th>Call Graph</th></tr>
</table><br />
</center></div>

    </td>
  </tr>
</table>
<!-- ==================== VARIABLES ==================== -->
<a name="section-Variables"></a>
<table class="summary" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr bgcolor="#70b0f0" class="table-header">
  <td align="left" colspan="2" class="table-header">
    <span class="table-header">Variables</span></td>
</tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="DEBUG"></a><span class="summary-name">DEBUG</span> = <code title="False">False</code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a href="pybindgen.typehandlers.ctypeparser.tokenizer-module.html#VALID_IDENTIFIER_CHARS" class="summary-name">VALID_IDENTIFIER_CHARS</a> = <code title="set(['$',
     '0',
     '1',
     '2',
     '3',
     '4',
     '5',
     '6',
..."><code class="variable-group">set([</code><code class="variable-quote">'</code><code class="variable-string">$</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">0</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">1</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">2</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">3</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">4</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">5</code><code class="variable-ellipsis">...</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a href="pybindgen.typehandlers.ctypeparser.tokenizer-module.html#HEX_DIGITS" class="summary-name">HEX_DIGITS</a> = <code title="set(['0',
     '1',
     '2',
     '3',
     '4',
     '5',
     '6',
     '7',
..."><code class="variable-group">set([</code><code class="variable-quote">'</code><code class="variable-string">0</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">1</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">2</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">3</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">4</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">5</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">6</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">7</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">8</code><code class="variable-quote">'</code><code class="variable-op">,</code><code class="variable-ellipsis">...</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a href="pybindgen.typehandlers.ctypeparser.tokenizer-module.html#INT_OR_FLOAT_DIGITS" class="summary-name">INT_OR_FLOAT_DIGITS</a> = <code title="set(['+',
     '-',
     '0',
     '1',
     '2',
     '3',
     '4',
     '5',
..."><code class="variable-group">set([</code><code class="variable-quote">'</code><code class="variable-string">+</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">-</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">0</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">1</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">2</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">3</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-quote">'</code><code class="variable-string">4</code><code class="variable-quote">'</code><code class="variable-op">, </code><code class="variable-ellipsis">...</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="UNKNOWN"></a><span class="summary-name">UNKNOWN</span> = <code title="'UNKNOWN'"><code class="variable-quote">'</code><code class="variable-string">UNKNOWN</code><code class="variable-quote">'</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="SYNTAX"></a><span class="summary-name">SYNTAX</span> = <code title="'SYNTAX'"><code class="variable-quote">'</code><code class="variable-string">SYNTAX</code><code class="variable-quote">'</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="CONSTANT"></a><span class="summary-name">CONSTANT</span> = <code title="'CONSTANT'"><code class="variable-quote">'</code><code class="variable-string">CONSTANT</code><code class="variable-quote">'</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="NAME"></a><span class="summary-name">NAME</span> = <code title="'NAME'"><code class="variable-quote">'</code><code class="variable-string">NAME</code><code class="variable-quote">'</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="PREPROCESSOR"></a><span class="summary-name">PREPROCESSOR</span> = <code title="'PREPROCESSOR'"><code class="variable-quote">'</code><code class="variable-string">PREPROCESSOR</code><code class="variable-quote">'</code></code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="WHENCE_QUEUE"></a><span class="summary-name">WHENCE_QUEUE</span> = <code title="1">1</code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="WHENCE_STREAM"></a><span class="summary-name">WHENCE_STREAM</span> = <code title="0">0</code>
    </td>
  </tr>
<tr>
    <td width="15%" align="right" valign="top" class="summary">
      <span class="summary-type">&nbsp;</span>
    </td><td class="summary">
        <a name="__package__"></a><span class="summary-name">__package__</span> = <code title="'pybindgen.typehandlers.ctypeparser'"><code class="variable-quote">'</code><code class="variable-string">pybindgen.typehandlers.ctypeparser</code><code class="variable-quote">'</code></code>
    </td>
  </tr>
</table>
<!-- ==================== FUNCTION DETAILS ==================== -->
<a name="section-FunctionDetails"></a>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr bgcolor="#70b0f0" class="table-header">
  <td align="left" colspan="2" class="table-header">
    <span class="table-header">Function Details</span></td>
</tr>
</table>
<a name="GetTokens"></a>
<div>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr><td>
  <table width="100%" cellpadding="0" cellspacing="0" border="0">
  <tr valign="top"><td>
  <h3 class="epydoc"><span class="sig"><span class="sig-name">GetTokens</span>(<span class="sig-arg">source</span>)</span>
  </h3>
  </td><td align="right" valign="top"
    ><span class="codelink"><a href="pybindgen.typehandlers.ctypeparser.tokenizer-pysrc.html#GetTokens">source&nbsp;code</a></span>&nbsp;
    <br /><span class="codelink"><a href="javascript:void(0);" onclick="toggleCallGraph('call_graph_for_gettokens-div');return false;">call&nbsp;graph</a></span>&nbsp;</td>
  </tr></table>
  <div style="display:none" id="call_graph_for_gettokens-div"><center>
<table border="0" cellpadding="0" cellspacing="0">
  <tr><td><center>  <map id="call_graph_for_gettokens" name="call_graph_for_gettokens">
<area shape="rect" href="pybindgen.typehandlers.ctypeparser-module.html#parse_type" title="parse_type()" alt="" coords="5,5,141,37"/>
<area shape="rect" href="pybindgen.typehandlers.ctypeparser.tokenizer-module.html#GetTokens" title="GetTokens()" alt="" coords="191,5,321,37"/>
<area shape="rect" href="pybindgen.typehandlers.ctypeparser.tokenizer.Token-class.html#__init__" title="Token.__init__()" alt="" coords="372,5,535,37"/>
</map>
  <img src="call_graph_for_gettokens.gif" alt='' usemap="#call_graph_for_gettokens" ismap="ismap" class="graph-without-title" />
</center></td></tr>
  <tr><th>Call Graph</th></tr>
</table><br />
</center></div>

  <p>Returns a sequence of Tokens.</p>
  <dl class="fields">
    <dt>Parameters:</dt>
    <dd><ul class="nomargin-top">
        <li><strong class="pname"><code>source</code></strong> - string of C++ source code.</li>
    </ul></dd>
    <dt>Returns:</dt>
        <dd>an iterator that yields the <a 
          href="pybindgen.typehandlers.ctypeparser.tokenizer.Token-class.html"
          class="link">Token</a>s in the source.</dd>
  </dl>
</td></tr></table>
</div>
<br />
<!-- ==================== VARIABLES DETAILS ==================== -->
<a name="section-VariablesDetails"></a>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr bgcolor="#70b0f0" class="table-header">
  <td align="left" colspan="2" class="table-header">
    <span class="table-header">Variables Details</span></td>
</tr>
</table>
<a name="VALID_IDENTIFIER_CHARS"></a>
<div>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr><td>
  <h3 class="epydoc">VALID_IDENTIFIER_CHARS</h3>
  
  <dl class="fields">
  </dl>
  <dl class="fields">
    <dt>Value:</dt>
      <dd><table><tr><td><pre class="variable">
<code class="variable-group">set([</code><code class="variable-quote">'</code><code class="variable-string">$</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">0</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">1</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">2</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">3</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">4</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">5</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">6</code><code class="variable-quote">'</code><code class="variable-op">,</code>
<code class="variable-ellipsis">...</code>
</pre></td></tr></table>
</dd>
  </dl>
</td></tr></table>
</div>
<a name="HEX_DIGITS"></a>
<div>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr><td>
  <h3 class="epydoc">HEX_DIGITS</h3>
  
  <dl class="fields">
  </dl>
  <dl class="fields">
    <dt>Value:</dt>
      <dd><table><tr><td><pre class="variable">
<code class="variable-group">set([</code><code class="variable-quote">'</code><code class="variable-string">0</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">1</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">2</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">3</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">4</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">5</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">6</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">7</code><code class="variable-quote">'</code><code class="variable-op">,</code>
<code class="variable-ellipsis">...</code>
</pre></td></tr></table>
</dd>
  </dl>
</td></tr></table>
</div>
<a name="INT_OR_FLOAT_DIGITS"></a>
<div>
<table class="details" border="1" cellpadding="3"
       cellspacing="0" width="100%" bgcolor="white">
<tr><td>
  <h3 class="epydoc">INT_OR_FLOAT_DIGITS</h3>
  
  <dl class="fields">
  </dl>
  <dl class="fields">
    <dt>Value:</dt>
      <dd><table><tr><td><pre class="variable">
<code class="variable-group">set([</code><code class="variable-quote">'</code><code class="variable-string">+</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">-</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">0</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">1</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">2</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">3</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">4</code><code class="variable-quote">'</code><code class="variable-op">,</code>
     <code class="variable-quote">'</code><code class="variable-string">5</code><code class="variable-quote">'</code><code class="variable-op">,</code>
<code class="variable-ellipsis">...</code>
</pre></td></tr></table>
</dd>
  </dl>
</td></tr></table>
</div>
<br />
<!-- ==================== NAVIGATION BAR ==================== -->
<table class="navbar" border="0" width="100%" cellpadding="0"
       bgcolor="#a0c0ff" cellspacing="0">
  <tr valign="middle">
  <!-- Home link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="pybindgen-module.html">Home</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Tree link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="module-tree.html">Trees</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Index link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="identifier-index.html">Indices</a>&nbsp;&nbsp;&nbsp;</th>

  <!-- Help link -->
      <th>&nbsp;&nbsp;&nbsp;<a
        href="help.html">Help</a>&nbsp;&nbsp;&nbsp;</th>

      <th class="navbar" width="100%"></th>
  </tr>
</table>
<table border="0" cellpadding="0" cellspacing="0" width="100%%">
  <tr>
    <td align="left" class="footer">
    Generated by Epydoc 3.0.1 on Sun Jul 12 17:23:07 2009
    </td>
    <td align="right" class="footer">
      <a target="mainFrame" href="http://epydoc.sourceforge.net"
        >http://epydoc.sourceforge.net</a>
    </td>
  </tr>
</table>

<script type="text/javascript">
  <!--
  // Private objects are initially displayed (because if
  // javascript is turned off then we want them to be
  // visible); but by default, we want to hide them.  So hide
  // them unless we have a cookie that says to show them.
  checkCookie();
  // -->
</script>
</body>
</html>
